package com.chance.cc.crawler.development.bootstrap.pcauto;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.junit.Test;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.internalDownload;

/**
 * @author lt
 * @version 1.0
 * @date 2021-01-11 13:48:17
 * @email okprog@sina.com
 */
public class PcAutoStart {

    private static final String domain = "pcauto";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }


    public static void main(String[] args) {
        //设置https协议访问
        System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2,SSLv3");
        //关键词 source record
        CrawlerRequestRecord keywordCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("pcauto_keywords",turnPageItem)
                .httpUrl("http://192.168.1.217:9599//v1/meta/"+domain+"/keys?site=adhocKw")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        //user agents source record
        CrawlerRequestRecord userAgentsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("taobao_user_agents", turnPageItem)
                .httpUrl("https://fake-useragent.herokuapp.com/browsers/0.1.11")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        //视频采集
        CrawlerRequestRecord videoCrawler = doVideoCrawler();
        //文章采集
        CrawlerRequestRecord articleCrawler = doArticleCrawler();
        //论坛-最新发布采集
        CrawlerRequestRecord forumReleaseCrawler = doForumReleaseCrawler();
        //论坛-最新回复采集
        CrawlerRequestRecord forumReplyCrawler = doForumReplyCrawler();
        //车家号-最新文章采集
        CrawlerRequestRecord cheArticleCrawler = doCheArticleCrawler();
        //车家号-最新视频采集
        CrawlerRequestRecord cheVideoCrawler = doCheVideoCrawler();
        //口碑采集
        CrawlerRequestRecord praiseCrawler = doPraiseCrawler();
        //经销商发文采集
        CrawlerRequestRecord dealerArticleCrawler = doDealerArticleCrawler();


        DevCrawlerController.builder()
                .triggerInfo(domain,domain + "_trigger",System.currentTimeMillis(),domain+ "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
                .fileResultPipeline("D:\\chance\\data\\pcauto\\pcauto_forum_reply_test.json",false)
//                .requestRecord(articleCrawler)  //more job
//                .requestRecord(videoCrawler)
//                .requestRecord(forumReleaseCrawler)
                .requestRecord(forumReplyCrawler)
//                .requestRecord(cheArticleCrawler)
//                .requestRecord(cheVideoCrawler)
//                .requestRecord(praiseCrawler)
//                .requestRecord(dealerArticleCrawler)
                .supportRecord(keywordCrawlerRecord)
//                .supportRecord(userAgentsCrawlerRecord)
                .crawlerThreadNum(3)
                .build()
                .start();
    }

    public static CrawlerRequestRecord doArticleCrawler(){
        String site = "article";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://www.pcauto.com.cn/nation/gckx/")
                .httpUrl("https://www.pcauto.com.cn/nation/gckx/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*2,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doVideoCrawler(){
        String site = "video";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://pcauto.pcvideo.com.cn/")
                .httpUrl("https://pcauto.pcvideo.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doForumReleaseCrawler(){
        String site = "forum_release";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://www.pcauto.com.cn/")
                .httpUrl("https://www.pcauto.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doForumReplyCrawler(){
        String site = "forum_reply";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://www.pcauto.com.cn/")
                .httpUrl("https://www.pcauto.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*590,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doCheArticleCrawler(){
        String site = "che_article";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://chejiahao.autohome.com.cn/?infotype=2")
                .httpUrl("https://chejiahao.autohome.com.cn/?infotype=2")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(1,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doCheVideoCrawler(){
        String site = "che_video";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://chejiahao.autohome.com.cn/?infotype=3")
                .httpUrl("https://chejiahao.autohome.com.cn/?infotype=3")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(1,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doPraiseCrawler(){
        String site = "praise";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain,CrawlerEnum.CrawlerRequestType.turnPage)
                .recordKey("https://www.pcauto.com.cn/")
                .httpUrl("https://www.pcauto.com.cn/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*30,null))
                .proxy(proxy)
                .needParsed(false)
                .needWashed(false)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    public static CrawlerRequestRecord doDealerArticleCrawler(){
        String site = "dealer";

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .recordKey("https://price.pcauto.com.cn/shangjia/c3/")
                .httpUrl("https://price.pcauto.com.cn/shangjia/c3/")
                .releaseTime(System.currentTimeMillis())
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24,null))
                .proxy(proxy)
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        return requestRecord;
    }

    @Test
    public void testDownload(){
        //https://bbs.pcauto.com.cn/forum-16365-1_postat.html
        HttpConfig httpConfig = HttpConfig.me("test");
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        String url = "https://c.api.weibo.com/2/statuses/count/biz.json?access_token=2.00o4_w1HrAaeYBedf38e38b8SnITmD&ids=4631746798159101&exclude_comment_like=1";
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(url);
        HttpPage httpPage = downloader.download(httpRequest, httpConfig);
        System.out.println(httpPage.getRawText());
    }
}
